{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "8d2441db-0ad8-40f2-b042-6bb96f184711",
   "metadata": {
    "id": "8d2441db-0ad8-40f2-b042-6bb96f184711"
   },
   "source": [
    "# Rag 从入门到精通：查询转换\n",
    "\n",
    "查询转换是一组专注于重写和/或修改检索问题的方法。\n",
    "\n",
    "## 环境\n",
    "\n",
    "`(1) 包`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "df89be8f-2c49-4f4f-9503-2bff0b08a67a",
   "metadata": {
    "id": "df89be8f-2c49-4f4f-9503-2bff0b08a67a",
    "tags": []
   },
   "outputs": [],
   "source": [
    "! pip install -q langchain_community tiktoken langchain-openai langchainhub  langchain\n",
    "! pip install -q chromadb==0.4.15\n",
    "! pip install -q beautifulsoup4\n",
    "! pip install -q langchain-nomic\n",
    "! pip install -q --upgrade httpx httpx-sse PyJWT\n",
    "! pip install -q --upgrade --quiet  dashscope"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5258de38-0cc0-4d9d-a5ca-6e750ebe6976",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-10-29T02:18:17.931638Z",
     "iopub.status.busy": "2024-10-29T02:18:17.931322Z",
     "iopub.status.idle": "2024-10-29T02:18:17.935187Z",
     "shell.execute_reply": "2024-10-29T02:18:17.934734Z",
     "shell.execute_reply.started": "2024-10-29T02:18:17.931617Z"
    },
    "id": "5258de38-0cc0-4d9d-a5ca-6e750ebe6976",
    "tags": []
   },
   "outputs": [],
   "source": [
    "from google.colab import userdata\n",
    "DASHSCOPE_API_KEY=userdata.get('DASHSCOPE_API_KEY')\n",
    "DEEPSEEK_API_KEY=userdata.get('DEEPSEEK_API_KEY')\n",
    "LANGCHAIN_API_KEY=userdata.get('LANGCHAIN_API_KEY')\n",
    "OPENAI_API_KEY=userdata.get('OPENAI_API_KEY')\n",
    "\n",
    "from langchain_community.embeddings import DashScopeEmbeddings\n",
    "import os\n",
    "os.environ[\"LANGCHAIN_PROJECT\"] = f\"RAG_虚拟文档\"\n",
    "os.environ['LANGCHAIN_TRACING_V2'] = 'true'\n",
    "os.environ['LANGCHAIN_ENDPOINT'] = 'https://api.smith.langchain.com'\n",
    "os.environ['LANGCHAIN_API_KEY'] = LANGCHAIN_API_KEY\n",
    "os.environ['USER_AGENT'] = 'myagent'\n",
    "os.environ[\"DASHSCOPE_API_KEY\"] = DASHSCOPE_API_KEY\n",
    "os.environ[\"DEEPSEEK_API_KEY\"] = DEEPSEEK_API_KEY\n",
    "os.environ[\"OPENAI_API_KEY\"] = OPENAI_API_KEY"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eda212d9-f885-45d4-8c1f-6c9f1656545e",
   "metadata": {
    "id": "eda212d9-f885-45d4-8c1f-6c9f1656545e"
   },
   "source": [
    "## 问题改写多重查询"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "201fd530-e428-47ad-84b2-4daeb735100f",
   "metadata": {
    "id": "201fd530-e428-47ad-84b2-4daeb735100f"
   },
   "source": [
    "文档基础处理，索引和检索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9d1b6e2b-dd76-410d-b870-23e02564a665",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-10-29T02:18:23.787032Z",
     "iopub.status.busy": "2024-10-29T02:18:23.786723Z",
     "iopub.status.idle": "2024-10-29T02:18:35.359000Z",
     "shell.execute_reply": "2024-10-29T02:18:35.358502Z",
     "shell.execute_reply.started": "2024-10-29T02:18:23.787011Z"
    },
    "id": "9d1b6e2b-dd76-410d-b870-23e02564a665",
    "tags": []
   },
   "outputs": [],
   "source": [
    "#### 索引 ####\n",
    "from langchain_nomic.embeddings import NomicEmbeddings\n",
    "# 加载文档\n",
    "import bs4\n",
    "from langchain_community.document_loaders import WebBaseLoader\n",
    "loader = WebBaseLoader(\n",
    "    web_paths=(\"https://baike.baidu.com/item/%E4%BA%BA%E5%B7%A5%E6%99%BA%E8%83%BD/9180?fr=ge_ala#9-10\",)\n",
    ")\n",
    "blog_docs = loader.load()\n",
    "\n",
    "\n",
    "# 拆分\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n",
    "    chunk_size=300,\n",
    "    chunk_overlap=50)\n",
    "\n",
    "splits = text_splitter.split_documents(blog_docs)\n",
    "\n",
    "# 索引\n",
    "embeddings=DashScopeEmbeddings(\n",
    "    model=\"text-embedding-v1\", dashscope_api_key=DASHSCOPE_API_KEY\n",
    ")\n",
    "from langchain_community.vectorstores import Chroma\n",
    "vectorstore = Chroma.from_documents(documents=splits,\n",
    "                                    embedding=embeddings)\n",
    "\n",
    "retriever = vectorstore.as_retriever(search_kwargs={\"k\": 5})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e081aaa2-15f6-4d3c-937e-13cdcdae0b1a",
   "metadata": {
    "id": "e081aaa2-15f6-4d3c-937e-13cdcdae0b1a"
   },
   "source": [
    "初始化语言模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b44d1d6-062e-4107-b295-a419c7b41137",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "execution": {
     "iopub.execute_input": "2024-10-29T02:18:39.655276Z",
     "iopub.status.busy": "2024-10-29T02:18:39.654839Z",
     "iopub.status.idle": "2024-10-29T02:18:42.230093Z",
     "shell.execute_reply": "2024-10-29T02:18:42.229551Z",
     "shell.execute_reply.started": "2024-10-29T02:18:39.655255Z"
    },
    "id": "5b44d1d6-062e-4107-b295-a419c7b41137",
    "outputId": "07731644-8e40-4292-846a-65f0a5bbbc5c",
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "你好！有什么我可以帮助你的吗？\n"
     ]
    }
   ],
   "source": [
    "# from langchain_community.chat_models import ChatZhipuAI\n",
    "# from langchain_core.messages import AIMessage, HumanMessage, SystemMessage\n",
    "\n",
    "# llm = ChatZhipuAI(model=\"glm-4-plus\",temperature=0.5)\n",
    "# messages = [\n",
    "#     AIMessage(content=\"Hi.\"),\n",
    "#     SystemMessage(content=\"Your role is a poet.\"),\n",
    "#     HumanMessage(content=\"Write a short poem about AI in four lines.\"),\n",
    "# ]\n",
    "\n",
    "# response = llm.invoke(messages)\n",
    "# print(response.content)  # Displays the AI-generated poem\n",
    "\n",
    "# from langchain_community.chat_models.tongyi import ChatTongyi\n",
    "# from langchain_core.messages import HumanMessage\n",
    "\n",
    "# llm = ChatTongyi(\n",
    "#     streaming=True,\n",
    "# )\n",
    "# res = llm.stream([HumanMessage(content=\"hi\")], streaming=True)\n",
    "# for r in res:\n",
    "#     print(\"chat resp:\", r)\n",
    "from langchain_openai import ChatOpenAI\n",
    "\n",
    "\n",
    "llm=ChatOpenAI(model=\"gpt-4o\")\n",
    "# llm = ChatOpenAI(\n",
    "#     model='deepseek-reasoner',\n",
    "#     openai_api_key=DEEPSEEK_API_KEY,\n",
    "#     openai_api_base='https://api.deepseek.com'\n",
    "# )\n",
    "\n",
    "response = llm.invoke(\"你好!\")\n",
    "print(response.content)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "76f1b6c5-faa9-404b-90c6-22d3b40169fa",
   "metadata": {
    "id": "76f1b6c5-faa9-404b-90c6-22d3b40169fa"
   },
   "source": [
    "### “领导” 的秘书，多重角度问题改写的prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "965de464-0c98-4318-9f9e-f8a597c8d5d6",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-10-29T02:18:46.007558Z",
     "iopub.status.busy": "2024-10-29T02:18:46.007254Z",
     "iopub.status.idle": "2024-10-29T02:18:46.012967Z",
     "shell.execute_reply": "2024-10-29T02:18:46.012498Z",
     "shell.execute_reply.started": "2024-10-29T02:18:46.007540Z"
    },
    "id": "965de464-0c98-4318-9f9e-f8a597c8d5d6",
    "tags": []
   },
   "outputs": [],
   "source": [
    "from langchain.prompts import ChatPromptTemplate\n",
    "\n",
    "# 多重查询：不同视角\n",
    "template = \"\"\"你是一名 AI 语言模型助手。你的任务是生成给定用户问题的五个不同版本，\n",
    "以从向量数据库中检索相关文档。通过生成用户问题的多个视角，你\n",
    "的目标是帮助用户克服基于距离的相似性搜索的一些限制。\n",
    "提供这些以换行符分隔的备选问题。原始问题：{question}\"\"\"\n",
    "prompt_perspectives = ChatPromptTemplate.from_template(template)\n",
    "\n",
    "from langchain_core.output_parsers import StrOutputParser\n",
    "from langchain_openai import ChatOpenAI\n",
    "\n",
    "generate_queries = (\n",
    "    prompt_perspectives\n",
    "    | llm\n",
    "    | StrOutputParser()\n",
    "    | (lambda x: x.split(\"\\n\"))\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4278739-e622-4348-9d20-ef4b41f42642",
   "metadata": {
    "id": "d4278739-e622-4348-9d20-ef4b41f42642"
   },
   "source": [
    "多重检索答案的后期处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f253520-386f-434b-8daa-d6dadb89eddb",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "execution": {
     "iopub.execute_input": "2024-10-29T02:18:49.580950Z",
     "iopub.status.busy": "2024-10-29T02:18:49.580621Z",
     "iopub.status.idle": "2024-10-29T02:18:55.068933Z",
     "shell.execute_reply": "2024-10-29T02:18:55.068475Z",
     "shell.execute_reply.started": "2024-10-29T02:18:49.580931Z"
    },
    "id": "4f253520-386f-434b-8daa-d6dadb89eddb",
    "outputId": "07999da8-096f-47bb-818b-1556df6f88cc",
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-6-3905d9ae5787>:10: LangChainBetaWarning: The function `loads` is in beta. It is actively being worked on, so the API may change.\n",
      "  return [loads(doc) for doc in unique_docs]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "23"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain.load import dumps, loads\n",
    "\n",
    "def get_unique_union(documents: list[list]):\n",
    "    \"\"\" 检索到的文档的唯一联合 \"\"\"\n",
    "    # 展平列表列表，并将每个文档转换为字符串\n",
    "    flattened_docs = [dumps(doc) for sublist in documents for doc in sublist]\n",
    "    # 获取独特的文档\n",
    "    unique_docs = list(set(flattened_docs))\n",
    "    # 返回\n",
    "    return [loads(doc) for doc in unique_docs]\n",
    "\n",
    "# 检索\n",
    "question = \"根据这篇百度百科链接，你能介绍一下人工智能目前在中国的现状吗？\"\n",
    "retrieval_chain = generate_queries | retriever.map() | get_unique_union\n",
    "docs = retrieval_chain.invoke({\"question\":question})\n",
    "len(docs)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2844b5bb-564d-47ce-a480-aad822b87c68",
   "metadata": {
    "id": "2844b5bb-564d-47ce-a480-aad822b87c68"
   },
   "source": [
    "把所有都串起来，完整的多重问题改写RAG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af6e74e8-ddae-4165-9e4b-0022ac125194",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 175
    },
    "execution": {
     "iopub.execute_input": "2024-10-29T02:19:01.053916Z",
     "iopub.status.busy": "2024-10-29T02:19:01.053593Z",
     "iopub.status.idle": "2024-10-29T02:19:28.102273Z",
     "shell.execute_reply": "2024-10-29T02:19:28.101798Z",
     "shell.execute_reply.started": "2024-10-29T02:19:01.053897Z"
    },
    "id": "af6e74e8-ddae-4165-9e4b-0022ac125194",
    "outputId": "cef3279b-5eab-4444-d0f5-3b3a02a79f7d",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      },
      "text/plain": [
       "'根据提供的百度百科文档内容，以下是中国人工智能（AI）发展现状的概述：\\n\\n### 1. **产业规模与增长**\\n- 2020年中国人工智能产业规模达**3031亿元**，同比增长15%，增速略高于全球平均水平。\\n- 至2022年，中国AI核心产业规模已突破**5000亿元**，企业数量从2016年的28万家增至2022年的**超60万家**，增长超114%。\\n- 2024年1月数据显示，中国人工智能企业数量已超过**4400家**（此处可能存在数据表述矛盾，需以最新统计为准）。\\n\\n### 2. **区域与技术优势**\\n- 产业主要集中在北京、上海、广东、浙江等科技资源密集省份。\\n- 在**人工智能芯片**、**深度学习软件架构**和**中文自然语言处理**领域取得显著进展，体现本土化技术突破。\\n\\n### 3. **政策与战略布局**\\n- **国家层面**：国务院印发《新一代人工智能发展规划》，提出面向2030年的发展目标，并设立**11个国家AI创新应用先导区**，覆盖长三角、京津冀、粤港澳、成渝等战略区域。\\n- **法律与伦理**：最高人民法院发布《关于规范和加强人工智能司法应用的意见》，强调技术应用的规范性与可信性。\\n\\n### 4. **科研与创新**\\n- 2023年科技部启动“人工智能驱动的科学研究（AI for Science）”专项，推动AI在生物、材料等基础科学领域的交叉应用。\\n- 2022年中国科协提出“如何实现可信可靠可解释人工智能技术路线”为前沿科学问题，关注技术伦理与可靠性。\\n\\n### 5. **挑战与治理**\\n- **大模型治理**：人工智能大模型带来的数据安全、伦理等问题引发重视，需通过法律法规和伦理框架平衡发展与风险。\\n- **国际合作与竞争**：中国外交部强调反对将AI技术作为“富国游戏”，呼吁避免技术垄断和发展鸿沟，体现对全球公平参与的关注。\\n\\n### 6. **社会影响与认可**\\n- 2024年“人工智能”当选“汉语盘点2024年度国际词”，反映其全球影响力及中国社会对AI发展的关注。\\n\\n### 总结\\n中国人工智能产业在政策支持、技术研发和产业应用上呈现高速发展态势，同时注重法律规范与伦理治理，致力于实现技术创新与风险防控的动态平衡。未来需进一步突破复杂任务处理的技术瓶颈，并加强国际合作以应对全球性挑战。'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from operator import itemgetter\n",
    "from langchain_openai import ChatOpenAI\n",
    "from langchain_core.runnables import RunnablePassthrough\n",
    "\n",
    "\n",
    "# RAG\n",
    "template = \"\"\"根据此上下文回答以下问题：\n",
    "\n",
    "{context}\n",
    "\n",
    "问题：{question}\n",
    "\"\"\"\n",
    "\n",
    "prompt = ChatPromptTemplate.from_template(template)\n",
    "\n",
    "final_rag_chain = (\n",
    "    {\"context\": retrieval_chain,\n",
    "     \"question\": itemgetter(\"question\")}\n",
    "    | prompt\n",
    "    | llm\n",
    "    | StrOutputParser()\n",
    ")\n",
    "\n",
    "final_rag_chain.invoke({\"question\":question})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "618afe4a-f1d6-433a-9d28-f1956c2b83ef",
   "metadata": {
    "id": "618afe4a-f1d6-433a-9d28-f1956c2b83ef"
   },
   "source": [
    "## 问题改写 （RAG融合-rerank)\n",
    "\n",
    "\n",
    "### 与上面一样的多重问题改写prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34e7075b-b80d-461d-9e2e-e05e29436f3e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-28T02:00:48.686385Z",
     "iopub.status.busy": "2024-10-28T02:00:48.686094Z",
     "iopub.status.idle": "2024-10-28T02:00:48.689653Z",
     "shell.execute_reply": "2024-10-28T02:00:48.689139Z",
     "shell.execute_reply.started": "2024-10-28T02:00:48.686370Z"
    },
    "id": "34e7075b-b80d-461d-9e2e-e05e29436f3e",
    "tags": []
   },
   "outputs": [],
   "source": [
    "from langchain.prompts import ChatPromptTemplate\n",
    "\n",
    "template = \"\"\"你是一名 AI 语言模型助手。你的任务是生成给定用户问题的五个不同版本，\n",
    "以从向量数据库中检索相关文档。通过生成用户问题的多个视角，你\n",
    "的目标是帮助用户克服基于距离的相似性搜索的一些限制。\n",
    "提供这些以换行符分隔的备选问题。原始问题：{question}\"\"\"\n",
    "prompt_rag_fusion = ChatPromptTemplate.from_template(template)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9781b40c-c408-42f4-ae14-cd11be513b63",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-10-28T02:00:58.799423Z",
     "iopub.status.busy": "2024-10-28T02:00:58.799149Z",
     "iopub.status.idle": "2024-10-28T02:00:58.803055Z",
     "shell.execute_reply": "2024-10-28T02:00:58.802607Z",
     "shell.execute_reply.started": "2024-10-28T02:00:58.799409Z"
    },
    "id": "9781b40c-c408-42f4-ae14-cd11be513b63",
    "tags": []
   },
   "outputs": [],
   "source": [
    "from langchain_core.output_parsers import StrOutputParser\n",
    "from langchain_openai import ChatOpenAI\n",
    "\n",
    "generate_queries = (\n",
    "    prompt_rag_fusion\n",
    "    | llm\n",
    "    | StrOutputParser()\n",
    "    | (lambda x: x.split(\"\\n\"))\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "658e2596-fe0c-4753-921b-0cf5bd233549",
   "metadata": {
    "id": "658e2596-fe0c-4753-921b-0cf5bd233549"
   },
   "source": [
    "招了一个文档处理的“专家”，给筛选出的文档做排名整合再呈现给“领导”"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b1adff1-e993-4747-b95d-656eaaeccfdd",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "execution": {
     "iopub.execute_input": "2024-10-28T02:02:15.220231Z",
     "iopub.status.busy": "2024-10-28T02:02:15.219938Z",
     "iopub.status.idle": "2024-10-28T02:02:18.601882Z",
     "shell.execute_reply": "2024-10-28T02:02:18.601491Z",
     "shell.execute_reply.started": "2024-10-28T02:02:15.220213Z"
    },
    "id": "2b1adff1-e993-4747-b95d-656eaaeccfdd",
    "outputId": "525e188a-4092-4b71-8476-521e6bc470f9",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain.load import dumps, loads\n",
    "\n",
    "question = \"根据这篇百度百科链接，你能介绍一下人工智能目前在中国的现状吗？\"\n",
    "\n",
    "def reciprocal_rank_fusion(results: list[list], k=60):\n",
    "    \"\"\" Reciprocal_rank_fusion 采用多个排名文档列表\n",
    "         以及 RRF 公式中使用的可选参数 k \"\"\"\n",
    "    print(f\"问题检索合并去重后结果：{results}\")\n",
    "    # 初始化一个字典来保存每个唯一文档的融合分数\n",
    "    fused_scores = {}\n",
    "    # 迭代每个排名文档列表\n",
    "    for docs in results:\n",
    "        # 迭代列表中的每个文档及其排名（列表中的位置）\n",
    "        # enumerate(docs):将 docs列表转换为 (index, value)元组的迭代器 list(enumerate([\"a\", \"b\", \"c\"]))  # 输出 [(0, \"a\"), (1, \"b\"), (2, \"c\")]\n",
    "        for rank, doc in enumerate(docs):\n",
    "            # 将文档转换为字符串格式以用作键（假设文档可以序列化为 JSON）\n",
    "            doc_str = dumps(doc)\n",
    "            # 如果文档尚未在 fused_scores 字典中，则将其添加，初始分数为 0\n",
    "            if doc_str not in fused_scores:\n",
    "                fused_scores[doc_str] = 0\n",
    "            # 检索文档的当前分数（如果有）\n",
    "            previous_score = fused_scores[doc_str]\n",
    "            # 使用 RRF 公式更新文档的分数：1 / (rank + k)\n",
    "            fused_scores[doc_str] += 1 / (rank + k)\n",
    "\n",
    "    # 根据融合分数对文档进行降序排序，得到最终的重新排序结果\n",
    "    reranked_results = [\n",
    "        (loads(doc), score)\n",
    "        for doc, score in sorted(fused_scores.items(), key=lambda x: x[1], reverse=True)\n",
    "    ]\n",
    "\n",
    "    # 将重新排序的结果作为元组列表返回，每个元组包含文档及其融合分数\n",
    "    return reranked_results\n",
    "\n",
    "retrieval_chain_rag_fusion = generate_queries | retriever.map() | reciprocal_rank_fusion\n",
    "docs = retrieval_chain_rag_fusion.invoke({\"question\": question})\n",
    "len(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ce2adf2d-3d9f-4d43-afb0-8304edcfb1f1",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 105
    },
    "execution": {
     "iopub.execute_input": "2024-10-28T02:02:26.800328Z",
     "iopub.status.busy": "2024-10-28T02:02:26.799851Z",
     "iopub.status.idle": "2024-10-28T02:02:55.378293Z",
     "shell.execute_reply": "2024-10-28T02:02:55.377906Z",
     "shell.execute_reply.started": "2024-10-28T02:02:26.800313Z"
    },
    "id": "ce2adf2d-3d9f-4d43-afb0-8304edcfb1f1",
    "outputId": "94dc880d-54e0-468c-b6e4-cba0c4bf8904",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      },
      "text/plain": [
       "'根据提供的百度百科链接内容，人工智能在中国的现状可以总结如下：\\n\\n1. **人工智能的广泛应用**：人工智能在中国被视为科技革命和产业变革的重要推动力量，涉及多个领域，包括机器人、语言识别、图像识别、自然语言处理、专家系统、机器学习以及计算机视觉等。人工智能已经成为智能学科的重要组成部分，目标在于模拟、延伸和扩展人的智能。\\n\\n2. **法律法规与伦理考量**：随着人工智能技术的快速发展，其带来的治理挑战也越来越受到重视。中国正在努力营造良好的创新生态，推动前瞻研究，并逐步建立保障人工智能健康发展的法律法规、制度体系及伦理道德框架。此外，还有建立容错、纠错机制以实现规范与发展的动态平衡。\\n\\n3. **国际竞争与合作**：中国强调人工智能是全人类的共同财富，而不应成为“富国和富人”的专属资源。中国外交部发言人曾指出一些国家在人工智能领域存在的分化和不平等问题。\\n\\n4. **公众意识与普及**：截至2024年底，中国有大量的人口听说过生成式人工智能产品，这表明人工智能的影响力和普及度在逐渐提高。\\n\\n5. **政策与规划**：中国在政策层面积极推动人工智能的发展。科技部会同自然科学基金委启动的“人工智能驱动的科学研究”（AI for Science）项目，旨在贯彻落实国家《新一代人工智能发展规划》，推动科学研究进一步与人工智能技术相结合。\\n\\n总体来说，人工智能在中国处于快速发展中，不仅在科技和产业方面取得了显著的进展，同时也在政策和法律层面积极推动其健康发展。'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain_core.runnables import RunnablePassthrough\n",
    "from operator import itemgetter\n",
    "# RAG\n",
    "template = \"\"\"根据此上下文回答以下问题：\n",
    "\n",
    "{context}\n",
    "\n",
    "问题：{question}\n",
    "\"\"\"\n",
    "\n",
    "prompt = ChatPromptTemplate.from_template(template)\n",
    "\n",
    "final_rag_chain = (\n",
    "    {\"context\": retrieval_chain_rag_fusion,\n",
    "     \"question\": itemgetter(\"question\")}\n",
    "    | prompt\n",
    "    | llm\n",
    "    | StrOutputParser()\n",
    ")\n",
    "question = \"根据这篇百度百科链接，你能介绍一下人工智能目前在中国的现状吗？\"\n",
    "final_rag_chain.invoke({\"question\":question})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "94c812d3-4d91-4634-8301-0b68be88a887",
   "metadata": {
    "id": "94c812d3-4d91-4634-8301-0b68be88a887"
   },
   "source": [
    "## 问题分解"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8b048319-5572-48d4-bd07-edebfcbbe198",
   "metadata": {
    "id": "8b048319-5572-48d4-bd07-edebfcbbe198"
   },
   "source": [
    "领导的新的“秘书”，分解问题的prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f82fac99-58dc-4bb9-84e6-51180db855ad",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-28T02:03:42.904480Z",
     "iopub.status.busy": "2024-10-28T02:03:42.904196Z",
     "iopub.status.idle": "2024-10-28T02:03:42.908073Z",
     "shell.execute_reply": "2024-10-28T02:03:42.907700Z",
     "shell.execute_reply.started": "2024-10-28T02:03:42.904465Z"
    },
    "id": "f82fac99-58dc-4bb9-84e6-51180db855ad",
    "tags": []
   },
   "outputs": [],
   "source": [
    "from langchain.prompts import ChatPromptTemplate\n",
    "\n",
    "# 分解\n",
    "template = \"\"\"您是一位乐于助人的助手，可以生成与输入问题相关的多个子问题。\\n\n",
    "目标是将输入分解为一组可以单独回答的子问题/子问题。\\n\n",
    "生成与以下问题相关的多个搜索查询：{question} \\n\n",
    "输出（5 个查询）：\"\"\"\n",
    "prompt_decomposition = ChatPromptTemplate.from_template(template)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c31eefd9-5598-44a1-b0d6-dd04553a3eb4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-28T02:03:58.886963Z",
     "iopub.status.busy": "2024-10-28T02:03:58.886699Z",
     "iopub.status.idle": "2024-10-28T02:04:02.512128Z",
     "shell.execute_reply": "2024-10-28T02:04:02.511716Z",
     "shell.execute_reply.started": "2024-10-28T02:03:58.886948Z"
    },
    "id": "c31eefd9-5598-44a1-b0d6-dd04553a3eb4",
    "tags": []
   },
   "outputs": [],
   "source": [
    "from langchain_openai import ChatOpenAI\n",
    "from langchain_core.output_parsers import StrOutputParser\n",
    "\n",
    "\n",
    "# Chain\n",
    "generate_queries_decomposition = ( prompt_decomposition | llm | StrOutputParser() | (lambda x: x.split(\"\\n\")))\n",
    "\n",
    "\n",
    "question = \"根据文章，中国的人工智能是怎么一步步发展到今天的？\"\n",
    "questions = generate_queries_decomposition.invoke({\"question\":question})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "07191b5c-cf72-4b8f-a225-f57dfdc2fc78",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "execution": {
     "iopub.execute_input": "2024-10-28T02:04:06.402534Z",
     "iopub.status.busy": "2024-10-28T02:04:06.402268Z",
     "iopub.status.idle": "2024-10-28T02:04:06.406487Z",
     "shell.execute_reply": "2024-10-28T02:04:06.406133Z",
     "shell.execute_reply.started": "2024-10-28T02:04:06.402519Z"
    },
    "id": "07191b5c-cf72-4b8f-a225-f57dfdc2fc78",
    "outputId": "67475837-20f5-4a85-8f74-f98d628e1298",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['1. 中国人工智能发展的历史节点是什么？',\n",
       " '2. 近年来中国在人工智能领域的重大突破有哪些？',\n",
       " '3. 中国的人工智能技术在各个行业中的应用情况如何？',\n",
       " '4. 中国政府在推动人工智能发展中发挥了怎样的作用？',\n",
       " '5. 中外在人工智能技术发展上的差异和比较有哪些？']"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "questions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19543d04-ff31-4774-b89c-9d31f5a28fc9",
   "metadata": {
    "id": "19543d04-ff31-4774-b89c-9d31f5a28fc9"
   },
   "source": [
    "### 招了另外一个“专家”，新的方式整合搜索结果给“领导”\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c72bbd12-f85c-4ed0-9dfa-8503afebfafa",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-28T02:05:07.200372Z",
     "iopub.status.busy": "2024-10-28T02:05:07.200108Z",
     "iopub.status.idle": "2024-10-28T02:05:07.203327Z",
     "shell.execute_reply": "2024-10-28T02:05:07.202894Z",
     "shell.execute_reply.started": "2024-10-28T02:05:07.200357Z"
    },
    "id": "c72bbd12-f85c-4ed0-9dfa-8503afebfafa",
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Prompt\n",
    "template = \"\"\"这是您需要回答的问题：\n",
    "\n",
    "\\n --- \\n {question} \\n --- \\n\n",
    "\n",
    "以下是任何可用的背景问题 + 答案组合：\n",
    "\n",
    "\\n --- \\n {q_a_pairs} \\n --- \\n\n",
    "\n",
    "以下是与问题相关的其他上下文：\n",
    "\n",
    "\\n --- \\n {context} \\n --- \\n\n",
    "\n",
    "使用上述上下文和任何背景问题 + 答案组合来回答问题：\\n {question}\n",
    "\"\"\"\n",
    "\n",
    "decomposition_prompt = ChatPromptTemplate.from_template(template)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "49c8173b-21c2-4d26-9e90-ed3bf8aacc7d",
   "metadata": {
    "id": "49c8173b-21c2-4d26-9e90-ed3bf8aacc7d"
   },
   "source": [
    "把所有东西都串联起来"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a20bf0d4-f567-4451-834d-a07190a3185e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-28T02:05:21.851845Z",
     "iopub.status.busy": "2024-10-28T02:05:21.851564Z",
     "iopub.status.idle": "2024-10-28T02:07:15.631787Z",
     "shell.execute_reply": "2024-10-28T02:07:15.631365Z",
     "shell.execute_reply.started": "2024-10-28T02:05:21.851830Z"
    },
    "id": "a20bf0d4-f567-4451-834d-a07190a3185e",
    "tags": []
   },
   "outputs": [],
   "source": [
    "from operator import itemgetter\n",
    "from langchain_core.output_parsers import StrOutputParser\n",
    "\n",
    "def format_qa_pair(question, answer):\n",
    "    \"\"\"格式化 Q 和 A 对\"\"\"\n",
    "\n",
    "    formatted_string = \"\"\n",
    "    formatted_string += f\"Question: {question}\\nAnswer: {answer}\\n\\n\"\n",
    "    return formatted_string.strip()\n",
    "\n",
    "\n",
    "q_a_pairs = \"\"\n",
    "for q in questions:\n",
    "\n",
    "    rag_chain = (\n",
    "    {\"context\": itemgetter(\"question\") | retriever,\n",
    "     \"question\": itemgetter(\"question\"),\n",
    "     \"q_a_pairs\": itemgetter(\"q_a_pairs\")}\n",
    "    | decomposition_prompt\n",
    "    | llm\n",
    "    | StrOutputParser())\n",
    "\n",
    "    answer = rag_chain.invoke({\"question\":q,\"q_a_pairs\":q_a_pairs})\n",
    "    q_a_pair = format_qa_pair(q,answer)\n",
    "    q_a_pairs = q_a_pairs + \"\\n---\\n\"+  q_a_pair"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e6070fea-ffcf-49ca-ac99-7d7ed2744d40",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 140
    },
    "execution": {
     "iopub.execute_input": "2024-10-28T02:07:31.667653Z",
     "iopub.status.busy": "2024-10-28T02:07:31.667389Z",
     "iopub.status.idle": "2024-10-28T02:07:31.671359Z",
     "shell.execute_reply": "2024-10-28T02:07:31.670928Z",
     "shell.execute_reply.started": "2024-10-28T02:07:31.667639Z"
    },
    "id": "e6070fea-ffcf-49ca-ac99-7d7ed2744d40",
    "outputId": "70c77cbc-f29e-464c-87dd-574a33087771",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      },
      "text/plain": [
       "'中外在人工智能技术发展上的差异和比较主要体现在以下几个方面：\\n\\n1. **政策与战略规划**：\\n   - **中国**：中国政府对人工智能发展高度重视，通过出台一系列政策和战略规划推动行业发展，2017年发布的《新一代人工智能发展规划》提出到2030年成为全球主要AI创新中心的目标。\\n   - **国外**：美国、欧盟等国家和地区也制定了相应的AI发展政策，例如美国的“AI国家战略”，但不同地区的侧重点和战略目标有所不同。\\n\\n2. **科研与投资**：\\n   - **中国**：近年来，中国在AI研究和开发方面的投入显著增加，有多家科技公司和研究机构进行广泛的AI研究。此外，政府的大量资金支持和项目资助也推动了AI技术的快速发展。\\n   - **国外**：美国凭借其强大的科技公司和高校科研能力，一直在AI领域保持领先地位；欧洲则更加关注AI的伦理和规则制定。\\n\\n3. **技术应用与产业化**：\\n   - **中国**：人工智能在中国的各个行业迅速应用，如互联网、金融、医疗、制造、交通等，推动了这些传统行业的智能化升级。\\n   - **国外**：在美国等地，AI技术在自动驾驶、医疗技术、金融服务等方面也有广泛应用，但在落地速度和规模上不一。\\n\\n4. **学术研究与人才培养**：\\n   - **中国**：虽然近年来中国的AI研究论文数量和质量迅速提升，但在顶尖人才培养和基础理论创新上与顶尖水平仍有差距。\\n   - **国外**：欧美国家拥有众多顶尖学府和科技公司，长期以来在AI基础理论和应用研究上占据优势。\\n\\n5. **伦理与法律框架**：\\n   - **中国**：中国正在加紧制定AI相关的伦理规范和法律框架，以规范技术发展和应用，同时推动国际合作。\\n   - **国外**：欧美更早开始讨论AI的法律法规及伦理问题，特别是在隐私保护和数据安全方面制定了详细的法规。\\n\\n总体而言，中外在人工智能技术发展上各有侧重与优势，中国在应用落地速度和市场规模上具有一定优势，而国外则在基础研究、伦理法律框架及顶尖人才积累方面更为突出。这种差异反映了不同国家在AI技术发展战略上的不同选择和侧重点。'"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "answer"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "LipKsztZDKV2",
   "metadata": {
    "id": "LipKsztZDKV2"
   },
   "source": [
    "## 第 8 部分：退一步"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "140d2aaa-88c2-4789-9afb-57b0eb6f174d",
   "metadata": {
    "id": "140d2aaa-88c2-4789-9afb-57b0eb6f174d"
   },
   "source": [
    "领导的又一位“秘书”，专门负责退一步总结客户需求"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1d74f9f2-543d-4e41-b90b-7bb527eca1d9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-28T02:08:54.589254Z",
     "iopub.status.busy": "2024-10-28T02:08:54.588986Z",
     "iopub.status.idle": "2024-10-28T02:08:54.593806Z",
     "shell.execute_reply": "2024-10-28T02:08:54.593305Z",
     "shell.execute_reply.started": "2024-10-28T02:08:54.589239Z"
    },
    "id": "1d74f9f2-543d-4e41-b90b-7bb527eca1d9",
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 一些镜头示例\n",
    "from langchain_openai import ChatOpenAI\n",
    "from langchain_core.output_parsers import StrOutputParser\n",
    "from langchain_core.prompts import ChatPromptTemplate, FewShotChatMessagePromptTemplate\n",
    "examples = [\n",
    "    {\n",
    "        \"input\": \"警察进行合法逮捕吗？\",\n",
    "        \"output\": \"警察可以做什么？\",\n",
    "    },\n",
    "    {\n",
    "        \"input\": \"Jan Sindel’s was born in what country?\",\n",
    "        \"output\": \"what is Jan Sindel’s personal history?\",\n",
    "    },\n",
    "]\n",
    "# 我们现在将它们转换为示例消息\n",
    "example_prompt = ChatPromptTemplate.from_messages(\n",
    "    [\n",
    "        (\"human\", \"{input}\"),\n",
    "        (\"ai\", \"{output}\"),\n",
    "    ]\n",
    ")\n",
    "few_shot_prompt = FewShotChatMessagePromptTemplate(\n",
    "    example_prompt=example_prompt,\n",
    "    examples=examples,\n",
    ")\n",
    "prompt = ChatPromptTemplate.from_messages(\n",
    "    [\n",
    "        (\n",
    "            \"system\",\n",
    "            \"\"\"您是世界知识方面的专家。您的任务是退一步思考，将问题解释为更通用的退一步思考问题，这样更容易回答。以下是几个示例：\"\"\",\n",
    "        ),\n",
    "        # 一些镜头示例\n",
    "        few_shot_prompt,\n",
    "        # 新的问题\n",
    "        (\"user\", \"{question}\"),\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8c9dc7ed-4b58-4c4f-ad24-717f51404ce1",
   "metadata": {
    "id": "8c9dc7ed-4b58-4c4f-ad24-717f51404ce1"
   },
   "source": [
    "测试下新的秘书的总结能力"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5cba100d-167f-4392-8f58-88729d3e4ce9",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 36
    },
    "execution": {
     "iopub.execute_input": "2024-10-28T02:09:02.739597Z",
     "iopub.status.busy": "2024-10-28T02:09:02.739338Z",
     "iopub.status.idle": "2024-10-28T02:09:03.671356Z",
     "shell.execute_reply": "2024-10-28T02:09:03.670925Z",
     "shell.execute_reply.started": "2024-10-28T02:09:02.739582Z"
    },
    "id": "5cba100d-167f-4392-8f58-88729d3e4ce9",
    "outputId": "025f9ccb-ffe8-485e-87cf-40fdcdc7f9b8",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      },
      "text/plain": [
       "'人工智能如何应用于数据分析和决策支持？'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain_openai import ChatOpenAI\n",
    "llm=ChatOpenAI(model=\"gpt-4o\")\n",
    "\n",
    "generate_queries_step_back = prompt | llm | StrOutputParser()\n",
    "question = \"人工智能如何用于金融风控？\"\n",
    "generate_queries_step_back.invoke({\"question\": question})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "999445b0-d8a0-4208-9bb6-38610667a00b",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 70
    },
    "execution": {
     "iopub.execute_input": "2024-10-28T02:09:21.874312Z",
     "iopub.status.busy": "2024-10-28T02:09:21.874036Z",
     "iopub.status.idle": "2024-10-28T02:15:19.868495Z",
     "shell.execute_reply": "2024-10-28T02:15:19.867643Z",
     "shell.execute_reply.started": "2024-10-28T02:09:21.874297Z"
    },
    "id": "999445b0-d8a0-4208-9bb6-38610667a00b",
    "outputId": "ad299535-f907-4b72-aa76-edc9a8f89942",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      },
      "text/plain": [
       "'人工智能在金融风控领域的应用包括但不限于以下几个方面：\\n1. 金融机构开始尝试将人工智能技术应用于风险防控领域，用科技创新来防范金融风险。\\n2. 腾讯发布了金融风控大模型，与中国信息通信研究院、中国科学技术大学、新加坡南洋理工大学、中原消费金融、微众银行等科研院校及金融机构联合制定了全球范围内首个金融风控领域的大模型国际标准。\\n3. 金融领域AI大模型呈现出“百花齐放”的景象，多家金融机构发布了金融大模型解决方案，为金融科技领域带来了创新和变革的可能。\\n4. 生成式人工智能技术在金融业中的应用尚处于技术探索和试点应用的并行期，预计未来会带动金融业生成式人工智能规模化应用。\\n\\n综上所述，人工智能在金融风控领域的应用涵盖了大模型技术、风险防控、标准制定等多个方面，为金融行业带来了创新和发展的机遇。'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain_core.runnables import RunnableLambda\n",
    "# 响应提示\n",
    "response_prompt_template = \"\"\"您是世界知识专家。我要问您一个问题。您的回答应全面，并且不与以下上下文相矛盾（如果它们相关）。否则，如果它们不相关，请忽略它们。\n",
    "\n",
    "# {normal_context}\n",
    "# {step_back_context}\n",
    "\n",
    "# 原始问题：{question}\n",
    "# 答案：\"\"\"\n",
    "response_prompt = ChatPromptTemplate.from_template(response_prompt_template)\n",
    "\n",
    "chain = (\n",
    "    {\n",
    "        # 使用普通问题检索上下文\n",
    "        \"normal_context\": RunnableLambda(lambda x: x[\"question\"]) | retriever,\n",
    "        # 使用后退问题检索上下文\n",
    "        \"step_back_context\": generate_queries_step_back | retriever,\n",
    "        # 把问题转过去\n",
    "        \"question\": lambda x: x[\"question\"],\n",
    "    }\n",
    "    | response_prompt\n",
    "    | ChatOpenAI(temperature=0)\n",
    "    | StrOutputParser()\n",
    ")\n",
    "\n",
    "chain.invoke({\"question\": question})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "63d0e558-4abe-42e4-a33a-2b93692f5fab",
   "metadata": {
    "id": "63d0e558-4abe-42e4-a33a-2b93692f5fab"
   },
   "source": [
    "## 第九部分: 虚拟文档嵌入HyDE\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "feb8460c-17c9-4003-82e2-83e6e3e32449",
   "metadata": {
    "id": "feb8460c-17c9-4003-82e2-83e6e3e32449"
   },
   "source": [
    "领导的又一位“秘书”，专门负责根据客户的提问扩展成一篇文章"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c2902575-bbbb-41a9-835b-9a24dc08261b",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 105
    },
    "execution": {
     "iopub.execute_input": "2024-10-28T02:15:48.136934Z",
     "iopub.status.busy": "2024-10-28T02:15:48.136678Z",
     "iopub.status.idle": "2024-10-28T02:16:10.224405Z",
     "shell.execute_reply": "2024-10-28T02:16:10.223985Z",
     "shell.execute_reply.started": "2024-10-28T02:15:48.136917Z"
    },
    "id": "c2902575-bbbb-41a9-835b-9a24dc08261b",
    "outputId": "80dead5c-bf69-4bdd-8c39-4a00ac3e3f5b",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      },
      "text/plain": [
       "'人工智能（AI）在中国的发展历程可以追溯到20世纪的中后期，但真正的快速增长始于21世纪初。以下是AI在中国的主要发展阶段：\\n\\n1. **起步阶段（1950s-1990s）**：\\n   在20世纪50年代，中国的科学家开始关注和探索人工智能的基本理论，并着手翻译和引入国外的AI研究成果。这一时期，中国的人工智能主要停留在理论研究层面，实践应用较少。\\n\\n2. **探索阶段（1990s-2000s）**：\\n   随着计算能力和算法的发展，特别是1990年代后期互联网的普及，中国的AI研究进入了探索阶段。高校和科研机构逐渐增多，政府也开始重视这一领域，投入了一定的研究资源。这一时期，中国的AI发展主要集中在语音识别、机器翻译等领域。\\n\\n3. **加速发展阶段（2010s）**：\\n   进入2010年代，随着深度学习的突破性进展，中国的人工智能进入了快速发展阶段。政府出台了一系列政策支持AI发展，如《新一代人工智能发展规划》。这一阶段，中国的科技公司和初创企业在智能语音、计算机视觉、自然语言处理等领域实现了显著进步。\\n\\n4. **全球引领阶段（2020s）**：\\n   在政策、资本和市场的加持下，中国的人工智能技术在全球范围内迅速崛起。中国涌现了一大批领先的AI公司，如百度、阿里巴巴、腾讯、华为等。这些公司不仅在AI技术上取得了重要突破，还在无人驾驶、智慧城市、医疗健康等多个领域实现了商业化应用。\\n\\n总之，中国在政府的支持与庞大的市场需求推动下，人工智能技术及其应用取得了显著的进展，不仅在国内市场占据重要地位，也在国际AI技术竞争中扮演了重要角色。随着技术的持续进步和政策的不断支持，中国人工智能的发展前景将更加广阔。'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain_openai import ChatOpenAI\n",
    "from langchain_core.output_parsers import StrOutputParser\n",
    "from langchain.prompts import ChatPromptTemplate\n",
    "\n",
    "# 虚拟文档生成\n",
    "template = \"\"\"请写一段介绍性文章来回答问题\n",
    "问题：{question}\n",
    "文章：\"\"\"\n",
    "prompt_hyde = ChatPromptTemplate.from_template(template)\n",
    "\n",
    "from langchain_core.output_parsers import StrOutputParser\n",
    "from langchain_openai import ChatOpenAI\n",
    "\n",
    "generate_docs_for_retrieval = (\n",
    "    prompt_hyde | llm | StrOutputParser()\n",
    ")\n",
    "\n",
    "question = \"AI在中国的发展历程\"\n",
    "generate_docs_for_retrieval.invoke({\"question\":question})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef657403-9c19-4e19-8ca8-9572a4a59e3d",
   "metadata": {
    "id": "ef657403-9c19-4e19-8ca8-9572a4a59e3d"
   },
   "source": [
    "看看经过扩展后的文章能搜索到什么相关信息"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d47587bb-23db-42a0-b087-beef9e95308b",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "execution": {
     "iopub.execute_input": "2024-10-28T02:16:58.101457Z",
     "iopub.status.busy": "2024-10-28T02:16:58.101189Z",
     "iopub.status.idle": "2024-10-28T02:17:27.189052Z",
     "shell.execute_reply": "2024-10-28T02:17:27.188495Z",
     "shell.execute_reply.started": "2024-10-28T02:16:58.101442Z"
    },
    "id": "d47587bb-23db-42a0-b087-beef9e95308b",
    "outputId": "ee2d967c-9c26-4df6-9739-f664d12f9fd9",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Document(metadata={'description': '人工智能（Artificial Intelligence），英文缩写为AI。是新一轮科技革命和产业变革的重要驱动力量，是研究、开发用于模拟、延伸和扩展人的智能的理论、方法、技术及应用系统的一门新的技术科学。人工智能是智能学科重要的组成部分，它企图了解智能的实质，并生产出一种新的能以与人类智能相似的方式做出反应的智能机器。人工智能是十分广泛的科学，包括机器人、语言识别、图像识别、自然语言处理、专家系统、机器学习，计算机视觉等。人工智能大模型带来的治理挑战也不容忽视。马斯克指出，在人工智能机器学习面具之下的本质仍然是统计。营造良好创新生态，需做好前瞻研究，建立健全保障人工智能健康发展的法律法规、制度体系、伦理道德。着眼未来，在重视防范风险的同时，也应同步建立容错、纠错机制，努力实现规范与发展的动态平衡。2024年12月20日，“人工智能”当选为汉语盘点2024年度国际词。当地时间2025年1月', 'language': 'No language found.', 'source': 'https://baike.baidu.com/item/%E4%BA%BA%E5%B7%A5%E6%99%BA%E8%83%BD/9180?fr=ge_ala#9-10', 'title': '人工智能（智能科学与技术专业术语）_百度百科'}, page_content='[4]在人工智能领域，2020年人工智能产业规模保持平稳增长，产业规模达到了3031亿元，同比增长15%，增速略高于全球的平均增速。产业主要集中在北京、上海、广东、浙江等省份，我国在人工智能芯片领域、深度学习软件架构领域、中文自然语言处理领域进展显著。'),\n",
       " Document(metadata={'description': '人工智能（Artificial Intelligence），英文缩写为AI。是新一轮科技革命和产业变革的重要驱动力量，是研究、开发用于模拟、延伸和扩展人的智能的理论、方法、技术及应用系统的一门新的技术科学。人工智能是智能学科重要的组成部分，它企图了解智能的实质，并生产出一种新的能以与人类智能相似的方式做出反应的智能机器。人工智能是十分广泛的科学，包括机器人、语言识别、图像识别、自然语言处理、专家系统、机器学习，计算机视觉等。人工智能大模型带来的治理挑战也不容忽视。马斯克指出，在人工智能机器学习面具之下的本质仍然是统计。营造良好创新生态，需做好前瞻研究，建立健全保障人工智能健康发展的法律法规、制度体系、伦理道德。着眼未来，在重视防范风险的同时，也应同步建立容错、纠错机制，努力实现规范与发展的动态平衡。2024年12月20日，“人工智能”当选为汉语盘点2024年度国际词。当地时间2025年1月', 'language': 'No language found.', 'source': 'https://baike.baidu.com/item/%E4%BA%BA%E5%B7%A5%E6%99%BA%E8%83%BD/9180?fr=ge_ala#9-10', 'title': '人工智能（智能科学与技术专业术语）_百度百科'}, page_content='[34]2024年12月20日，“人工智能（AI）”当选为汉语盘点2024年度国际词。 [61]发展现状播报编辑2021年7月13日，中国互联网协会发布了《中国互联网发展报告（2021）》。《报告》显示，2020年，人工智能产业规模达到了3031亿元。'),\n",
       " Document(metadata={'description': '人工智能（Artificial Intelligence），英文缩写为AI。是新一轮科技革命和产业变革的重要驱动力量，是研究、开发用于模拟、延伸和扩展人的智能的理论、方法、技术及应用系统的一门新的技术科学。人工智能是智能学科重要的组成部分，它企图了解智能的实质，并生产出一种新的能以与人类智能相似的方式做出反应的智能机器。人工智能是十分广泛的科学，包括机器人、语言识别、图像识别、自然语言处理、专家系统、机器学习，计算机视觉等。人工智能大模型带来的治理挑战也不容忽视。马斯克指出，在人工智能机器学习面具之下的本质仍然是统计。营造良好创新生态，需做好前瞻研究，建立健全保障人工智能健康发展的法律法规、制度体系、伦理道德。着眼未来，在重视防范风险的同时，也应同步建立容错、纠错机制，努力实现规范与发展的动态平衡。2024年12月20日，“人工智能”当选为汉语盘点2024年度国际词。当地时间2025年1月', 'language': 'No language found.', 'source': 'https://baike.baidu.com/item/%E4%BA%BA%E5%B7%A5%E6%99%BA%E8%83%BD/9180?fr=ge_ala#9-10', 'title': '人工智能（智能科学与技术专业术语）_百度百科'}, page_content='[15]艾媒咨询数据显示，2016年，中国人工智能产业规模已突破100亿元，增长率达到43.3%。2023年2月，工信部发布的数据表明，2022年中国AI核心产业规模已达到5000亿。2023年，合合信息旗下启信宝联合城市进化论发布发布的《中国人工智能产业图鉴》数据显示，2016年全国AI相关存续企业近28万家，20'),\n",
       " Document(metadata={'description': '人工智能（Artificial Intelligence），英文缩写为AI。是新一轮科技革命和产业变革的重要驱动力量，是研究、开发用于模拟、延伸和扩展人的智能的理论、方法、技术及应用系统的一门新的技术科学。人工智能是智能学科重要的组成部分，它企图了解智能的实质，并生产出一种新的能以与人类智能相似的方式做出反应的智能机器。人工智能是十分广泛的科学，包括机器人、语言识别、图像识别、自然语言处理、专家系统、机器学习，计算机视觉等。人工智能大模型带来的治理挑战也不容忽视。马斯克指出，在人工智能机器学习面具之下的本质仍然是统计。营造良好创新生态，需做好前瞻研究，建立健全保障人工智能健康发展的法律法规、制度体系、伦理道德。着眼未来，在重视防范风险的同时，也应同步建立容错、纠错机制，努力实现规范与发展的动态平衡。2024年12月20日，“人工智能”当选为汉语盘点2024年度国际词。当地时间2025年1月', 'language': 'No language found.', 'source': 'https://baike.baidu.com/item/%E4%BA%BA%E5%B7%A5%E6%99%BA%E8%83%BD/9180?fr=ge_ala#9-10', 'title': '人工智能（智能科学与技术专业术语）_百度百科'}, page_content='个人电脑和众多技术杂志使计算机技术展现在人们面前.有了像美国人工智能协会这样的基金会.因为AI开发 的需要，还出现了一阵研究人员进入私人公司的热潮。150多所像DEC（它雇了700多员工从事AI研究）这样的公司共花了10亿美元在内部的AI开发组上.其它AI领域也在80年代进入市场.其中一项就是机器视觉.'),\n",
       " Document(metadata={'description': '人工智能（Artificial Intelligence），英文缩写为AI。是新一轮科技革命和产业变革的重要驱动力量，是研究、开发用于模拟、延伸和扩展人的智能的理论、方法、技术及应用系统的一门新的技术科学。人工智能是智能学科重要的组成部分，它企图了解智能的实质，并生产出一种新的能以与人类智能相似的方式做出反应的智能机器。人工智能是十分广泛的科学，包括机器人、语言识别、图像识别、自然语言处理、专家系统、机器学习，计算机视觉等。人工智能大模型带来的治理挑战也不容忽视。马斯克指出，在人工智能机器学习面具之下的本质仍然是统计。营造良好创新生态，需做好前瞻研究，建立健全保障人工智能健康发展的法律法规、制度体系、伦理道德。着眼未来，在重视防范风险的同时，也应同步建立容错、纠错机制，努力实现规范与发展的动态平衡。2024年12月20日，“人工智能”当选为汉语盘点2024年度国际词。当地时间2025年1月', 'language': 'No language found.', 'source': 'https://baike.baidu.com/item/%E4%BA%BA%E5%B7%A5%E6%99%BA%E8%83%BD/9180?fr=ge_ala#9-10', 'title': '人工智能（智能科学与技术专业术语）_百度百科'}, page_content='[12]2023年3月，为贯彻落实国家《新一代人工智能发展规划》，科技部会同自然科学基金委启动“人工智能驱动的科学研究”（AI for')]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 检索\n",
    "retrieval_chain = generate_docs_for_retrieval | retriever\n",
    "retireved_docs = retrieval_chain.invoke({\"question\":question})\n",
    "retireved_docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "604fcc36-a1d7-4096-99b5-50db30950fc5",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 88
    },
    "execution": {
     "iopub.execute_input": "2024-10-28T02:17:36.507834Z",
     "iopub.status.busy": "2024-10-28T02:17:36.507572Z",
     "iopub.status.idle": "2024-10-28T02:17:57.938984Z",
     "shell.execute_reply": "2024-10-28T02:17:57.938605Z",
     "shell.execute_reply.started": "2024-10-28T02:17:36.507819Z"
    },
    "id": "604fcc36-a1d7-4096-99b5-50db30950fc5",
    "outputId": "31644de3-8c35-4ab2-c9d3-67f52cd63f9d",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      },
      "text/plain": [
       "'根据提供的上下文，中国的人工智能发展历程可以概述如下：\\n\\n1. **早期发展**：在1980年代，人工智能开始进入市场和研究领域。像DEC等公司已经在AI研究上投入了大量资金。\\n\\n2. **2016年**：人工智能产业规模突破100亿元，增长率达到43.3%。这一时期显示了AI产业的初步快速发展。\\n\\n3. **2020年**：AI产业规模达到了3031亿元，同比增长15%。产业主要集中在北京、上海、广东、浙江等省份，中国在AI领域的某些方面，如人工智能芯片、深度学习软件架构和中文自然语言处理领域取得显著进展。\\n\\n4. **2022年**：AI核心产业规模达到5000亿，显示了持续的增长和扩展。\\n\\n5. **政策支持**：2023年3月，为了贯彻落实国家《新一代人工智能发展规划》，科技部会同自然科学基金会启动了“人工智能驱动的科学研究”计划，旨在推动AI技术在多个领域的应用和发展。\\n\\n通过这些数据可以看出，中国的人工智能产业在政策支持、技术发展和市场扩展等方面都取得了显著进展，并呈现出快速增长的趋势。'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# RAG\n",
    "template = \"\"\"根据此上下文回答以下问题：\n",
    "\n",
    "{context}\n",
    "\n",
    "问题：{question}\n",
    "\"\"\"\n",
    "\n",
    "prompt = ChatPromptTemplate.from_template(template)\n",
    "\n",
    "final_rag_chain = (\n",
    "    prompt\n",
    "    | llm\n",
    "    | StrOutputParser()\n",
    ")\n",
    "\n",
    "final_rag_chain.invoke({\"context\":retireved_docs,\"question\":question})"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
